/**
 * author:Administrator
 * date:2016/4/12
 * desc:从网站上抓取图片
 * 基本思路是:
 * 先抓取网页，然后分析，
 * a标签    --------- 下一个url地址
 * img标签  ---------  因为我们要抓图片
 * 然后对 a标签、img标签 的结果做一些分析，过滤，
 * 然后 对image进行下载，把 a标签中的url 放到抓取列表中，循环处理抓取列表
 */
'use strict';

// lib
const request = require('request');
const urllib = require('url');
const cheerio = require('cheerio');
const fs = require('fs');
const moment = require('moment');
const EventEmitter = require('events').EventEmitter;
const crypto = require('crypto');
const Worker = require('./CrawlerWorker.js');
const path = require('path');

const HTTP_STR = 'http://';
const HTTP_LEN = HTTP_STR.length;

class MyCrawler extends EventEmitter {

  /**
   * @param opt
   * @param opt.debug = false|true
   * @param opt.saveFile = []
   * @param opt.url = []
   * @param opt.exclude = []
   * @param opt.include = []
   * @param opt.depth = 1
   * @param opt.maxThread = 1
   * @param opt.maxSize = 1
   * @param opt.minSize = 1
   * @param opt.label = 1
   */
  constructor(opt) {
    super();
    this.opt = opt;
    this.debug = opt.debug || false;
    this.sleepTime = 0;
    this.listTodo = [];
    this.listBlock = [];
    this.listFail = [];
    this.listDone = [];
    this.workingThread = 0;
    this.postfixSet = new Set();

    this.logDoneFile = path.join(__dirname, './log_done.txt');
    this.logFailFile = path.join(__dirname, './log_fail.txt');
    this.logPostfixFile = path.join(__dirname, './log_postfix.txt');
    this.logStaticFile = path.join(__dirname, './log_static.txt');

    //
    this.saveFile = opt.saveFile || [];
    this.exclude = opt.exclude || [];
    this.include = opt.include || [];
    this.maxDepth = opt.depth || -1;
    this.maxThread = opt.maxThread || 3;
    this.maxSize = opt.maxSize || 0;
    this.minSize = opt.minSize || 0;
    this.label = opt.label || {a: 'href'};

    //
    this.statistic = {};          // 统计数据
    this.limitAtMemory = 2000;     // 限制内存
  }

  /**
   * url生成器:抓取 a 标签，解析href
   */
  urlGen(url, html) {
    let self = this;
    let $ = cheerio.load(html);
    let rPathSet = new Set();

    let labelsName = Object.keys(this.label);
    for (let labelName of labelsName) {
      $(labelName).each(function () {
        let attrVal = $(this).attr(self.label[labelName]);
        if (attrVal) rPathSet.add(attrVal);
      });
    }

    let arr = Array.from(rPathSet);

    /**
     * 对url进行过滤 filter
     * 必须在 include之中
     * 在 exclude之外
     */
    if (self.include.length > 0) {
      arr = arr.filter(item => {
        return self.include.some(regex => regex.exec(item));
      });
    }
    if (self.exclude.length > 0) {
      arr = arr.filter(item => {
        return !self.exclude.some(regex => regex.exec(item));
      });
    }

    // console.log('rPathSet', rPathSet);
    // console.log('after filter, arr', arr);
    arr = arr.map(item => urllib.resolve(url, item));
    // console.log('urlGen', arr);
    return arr;
  }

  mkdir() {
    let folder = moment(new Date()).format('YYYY-MM-DD');
    let dir = path.join(__dirname, folder);
    try {
      let stats = fs.lstatSync(folder);
      if (stats.isDirectory()) {
      }
    } catch (e) {
      console.log('dir %s not exists make it', folder);
      fs.mkdirSync(folder, 0o777);
    }
    console.log('folder=', dir);
    this.folder = dir;
  }

  urlHandler(urls, preTask) {
    let self = this;
    let preDepth = preTask.depth;
    let newTasks = [];
    urls.forEach(url => {
      let aTask = {depth: preDepth + 1, url: url};
      let fileName = self.getFileName(url);
      if (fileName) {
        let postfix = fileName.postfix;
        if (!self.postfixSet.has(postfix)) {
          self.logPostfix(postfix);
          self.postfixSet.add(postfix);
        }
        if (self.saveFile.indexOf(fileName.postfix) >= 0) {
          aTask.filePath = path.join(self.folder, fileName.lastPath);
          aTask.minSize = self.minSize;
          aTask.maxSize = self.maxSize;
        }
      }
      newTasks.push(aTask);
    });
    // console.log('newTasks=', newTasks);
    return newTasks;
  }

  /**
   * 获取文件类型
   * @param url
   */
  getFileName(url) {
    let tmp = url.substring(HTTP_LEN);
    let pos = tmp.lastIndexOf('/');
    if (pos < 0) return null;
    let lastPath = tmp.split('/').pop();
    let rName = lastPath.split('.');
    // console.log('lastPath', lastPath);
    // console.log('rName', rName);
    if (rName.length <= 1)return null;
    let postfix = rName.pop();
    // 检测query string
    let qsIndex = postfix.indexOf('?');
    if (qsIndex >= 0) {
      postfix = postfix.substr(0, qsIndex);
    }
    return {
      postfix: postfix,        // 后缀名     jpg
      lastPath: lastPath       // 相对路径名  aa.jpg
    }
  }

  addTask(tasks) {
    let self = this;
    let addCnt = 0;
    tasks.forEach(task => {
      // depth
      if (self.maxDepth > 0 && self.maxDepth < task.depth) {
        return;
      }

      let urlMd5 = crypto.createHash('md5').update(task.url).digest('hex');
      let checkObj = urlMd5; //task.url
      if (self.listBlock.indexOf(checkObj) < 0) {
        self.listTodo.push(task);
        self.listBlock.push(checkObj);
        addCnt++;
      }
    });
    // console.log('addTask ', addCnt);
  }

  /**
   * 统计内存情况
   */
  doStatistic() {
    let memInfo = process.memoryUsage();
    this.statistic.memInfo = {
      rss: (memInfo.rss >> 20) + ' M',
      heapTotal: (memInfo.heapTotal >> 20) + ' M',
      heapUsed: (memInfo.heapUsed >> 20) + ' M'
    };
    if (this.debug) console.log(this.statistic.memInfo);
    this.logStatic(JSON.stringify(this.statistic.memInfo));
  }


  /**
   * 进行下一个task
   * @param worker
   */
  startNextTask(worker) {
    let self = this;

    /** case : memory too much, sleep this thread */
    let heapTotal = self.statistic.memInfo && self.statistic.memInfo.heapTotal.split(' ').shift();
    heapTotal = heapTotal ? +heapTotal : 0;
    if (heapTotal > self.limitAtMemory && self.workingThread > 0) {
      self.sleepTime += 100;
      if (this.debug)
        console.log('memory limit, sleep,worker.id=', worker.id);
      setTimeout(function () {
        self.startNextTask(worker);
      }, self.sleepTime);
      return;
    }

    /** case : all task done */
    if (self.listTodo.length == 0 && self.workingThread == 0) {
      self.gracefulQuit();
      return;
    }

    /** case : current task num =0 , wait*/
    if (self.listTodo.length == 0) {
      self.sleepTime += 100;
      if (this.debug)
        console.log('zero task sleep,worker.id=', worker.id);
      setTimeout(function () {
        self.startNextTask(worker);
      }, self.sleepTime);
      return;
    }

    /** case : have task */
    self.sleepTime -= 100;
    let item = self.listTodo.shift();
    self.workingThread++;
    worker.start(item);
  }

  logDone(content) {
    let self = this;
    fs.appendFile(self.logDoneFile, content + '\n', {encoding: 'utf8'}, function (err) {
    });
  }

  logFail(content) {
    let self = this;
    fs.appendFile(self.logFailFile, content + '\n', {encoding: 'utf8'}, function (err) {
    });
  }

  logPostfix(content) {
    let self = this;
    fs.appendFile(self.logPostfixFile, content + '\n', {encoding: 'utf8'}, function (err) {
    });
  }

  logStatic(content) {
    let self = this;
    fs.appendFile(self.logStaticFile, content + '\n', {encoding: 'utf8'}, function (err) {
    });
  }

  /**
   * task 完成
   * @param worker
   */
  taskDone(worker) {
    let self = this;
    self.workingThread--;

    let doneTask = worker.getTask();
    if (worker.error) {
      console.error('error', worker.error);
      self.listFail.push(doneTask.url);
      self.logFail(doneTask.url);
    } else {
      self.logDone(doneTask.url);
      self.listDone.push(doneTask.url);       // TODO
      if (doneTask.html) {
        let urls = self.urlGen(doneTask.url, doneTask.html);
        let tasks = self.urlHandler(urls, doneTask);
        self.addTask(tasks);
      }
    }

    setTimeout(function () {
      self.startNextTask(worker);
    }, 100);
  }

  start() {
    let self = this;
    self.init();
    self.mkdir();
    self.handleQuit();

    self.statistic.startAt = new Date();
    for (let i = 1; i <= self.maxThread; i++) {
      let worker = new Worker(i, {debug: self.debug});
      (function (theWorker) {
        theWorker.on('finish', function () {
          self.taskDone(theWorker);
        });
        self.startNextTask(theWorker);
      })(worker);
    }

    self.staticTask = setInterval(self.doStatistic.bind(self), 3000);
  }

  init() {
    let initUrl = this.opt.url;
    this.addTask([{depth: 0, url: initUrl}]);
  }

  gracefulQuit() {
    let self = this;
    if (self.printDone) return;
    self.printDone = true;
    // clean
    clearInterval(self.staticTask);

    //
    console.log('all done');
    self.statistic.endAt = new Date();
    console.log('time cost = ', self.statistic.endAt.getTime() - self.statistic.startAt.getTime());
    console.log('self.listDone.length = ', self.listDone.length);
    console.log('self.listFail.length =', self.listFail.length);
    console.log('self.postfixSet.length', self.postfixSet.size);
  }

  handleQuit() {
    let self = this;
    if (process.platform === "win32") {
      var rl = require("readline").createInterface({
        input: process.stdin,
        output: process.stdout
      });

      rl.on("SIGINT", function () {
        process.emit("SIGINT");
      });
    }

    process.on("SIGINT", function () {
      //graceful shutdown
      console.log('SIGINT');
      self.gracefulQuit();
      process.exit();
    });
  }
}

module.exports = MyCrawler;